package org.brian.webscrape;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import math.MathUtils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class CLScrape {
	
	//Q to answer: How many housing posts per day?
	// What is median price? apts vs rooms?
	
	//eyeballing: 50 posts/hr (at peak usage?)
	
	/**
	 * 0-1k: 11 ~ 15%
	 * 1-2k: 47 ~ 65%
	 * 2-3k: 13 ~ 18%
	 * 3-4k: 1  ~ 1%
	 */
	

	private static final Pattern PRICE_PATTERN = Pattern.compile("^\\$(\\d+) ");
	
	private static final Pattern ROOMS_PATTERN = Pattern.compile("^\\$(\\d+) / (\\d)br");
	
	private static final Pattern LOCATION_DESC_PATTERN = Pattern.compile("^\\((\\w+\\s*\\w*)\\)$");
	
	private static final Pattern ALL_PATTERN = Pattern.compile("(\\d+) / (\\d+)br - (\\d+)ft");
	
	private static final SimpleDateFormat CL_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd, hh:mma z");
	
	public static final String CL_CITY = "honolulu";
	
	public static final String OAHU_APTS = "http://sandiego.craigslist.org/nsd/apa";
	
	private List<CLPost> posts = new ArrayList<CLPost>();
	
	public static void main(String [] args) throws IOException, ParseException{
		
		long startTime = System.currentTimeMillis();
		String myUrl = "http://honolulu.craigslist.org/apa";
		
		CLScrape scraper = new CLScrape();
		
		int hundredsToProcess=1;
		scraper.quickScrape(myUrl);

		for (int i=hundredsToProcess; i>0 ; i--){
			scraper.quickScrape(myUrl+ "/index"+i+"00.html");
		}
		
		
		//System.out.println("");
		System.out.println("N: "+scraper.getNumberOfPosts() + " mean: "+scraper.getMeanPrice() + " median: "+ scraper.getMedian()+ " mode: "+scraper.getMode());
				
		//Persist p = new Persist();
		//p.save(scraper.posts);
			
		System.out.println("time taken: "+ (System.currentTimeMillis() - startTime)/1000 + " secs");
	}
	
	public int getNumberOfPosts(){
		return posts.size();		
	}
	
	public Number getMedian(){
		
		return MathUtils.median(buildPriceList());
	}
	
	public int getMeanPrice(){
		int total = 0;
		for(CLPost post : posts){
			if(post.getBedrooms() > 0)
				total+= post.getPrice() / post.getBedrooms();
		}
		
		
		
		return total / posts.size();
	}
	
	private List<Integer> buildPriceList(){
		List<Integer> prices = new ArrayList<Integer>();
		
		for(CLPost post : posts){
			Integer price = post.getPrice();
			Integer beds = post.getBedrooms();
			if(beds == null || 0 == beds){
				beds = 1;
			}
			System.out.println("price: "+price+" beds:"+beds+" p/b:"+price/beds);
			prices.add(price/beds);
		}
		return prices;
	}
	
	public Integer getMode(){
	
		return MathUtils.mode(buildPriceList());
	}
	
	public void quickScrape(String url){
		Document doc = getDocForUrl(url);
		buildPostList(doc);
			//System.out.println("Location: "+location);
			//System.out.println("Link: "+link);
			//System.out.println("Bedrooms : "+bedrooms);
			//System.out.println("Price: "+price);
	}
	
	private void buildPostList(Document doc){
		Elements rows = doc.getElementsByClass("row");
		for(Element row : rows){
			Integer price = 0;
			Integer bedrooms = null;
			String location = "";
			String link="";
			String locationHtml = "";
			Integer footage = 0;
			
			
			final String priceBrsHtml = row.getElementsByClass("itemph").html();			
			link = row.getElementsByTag("a").attr("href");
						
			/*
			Elements locationElements = row.getElementsByClass("itempn");
			if(locationElements.size() > 0){
				Element ele = locationElements.get(0);
				locationHtml = ele.getElementsByTag("font").html();
			}
			*/			
			
			Matcher oneMatcher = ALL_PATTERN.matcher(priceBrsHtml);
			while(oneMatcher.find()){
				if(oneMatcher.groupCount() == 3){
					price = Integer.valueOf(oneMatcher.group(1));
					bedrooms = Integer.valueOf(oneMatcher.group(2));
					footage = Integer.valueOf(oneMatcher.group(3));
					//System.out.print("Price: "+oneMatcher.group(1)+ " brs: "+oneMatcher.group(2)+ " sq-ft:"+oneMatcher.group(3));
					//System.out.println("");
					posts.add( new CLPost(null, link, price, bedrooms,location, footage));
				}
			}
			
			/*
			Matcher matcher = PRICE_PATTERN.matcher(priceBrsHtml);
			while(matcher.find()){
				price = matcher.group(1);				
			}
			
			Matcher rmMatcher = ROOMS_PATTERN.matcher(priceBrsHtml);
			while(rmMatcher.find()){
				 String text = rmMatcher.group(2);
				 if(text != null){
					 bedrooms = Integer.valueOf(text);
				 }
			}
			

			Matcher locMatcher = LOCATION_DESC_PATTERN.matcher(locationHtml);
			while(locMatcher.find()){
				location = locMatcher.group(1);
			}	
			*/

			

		}
	}
	
	public void scrape(String url) throws IOException, ParseException{
		Document doc = getDocForUrl(url);
		Elements rows = doc.getElementsByAttributeValueContaining("href", CL_CITY);
		
		for(Element rowElement : rows){
			CLPost p = buildPost(rowElement);
			if(p.getPrice() != null){
				System.out.println(p);
				posts.add(p);
			}
	
		}				
		
	}
	
	
	private CLPost buildPost(Element e) throws IOException, ParseException{
		String link = e.attr("href");
		String price = null;
		Integer bedrooms = null;
		Date date = null;
		
		
		Document page = getDocForUrl(link);
		Elements titles = page.getElementsByClass("postingtitle");
		for(Element titleEle : titles){
			String val = titleEle.text();				
			
			Matcher matcher = PRICE_PATTERN.matcher(val);
			while(matcher.find()){
				price = matcher.group(1);				
			}
			Matcher rmMatcher = ROOMS_PATTERN.matcher(val);
			while(rmMatcher.find()){
				String brs = rmMatcher.group(2);
				if(null != brs){
					bedrooms = Integer.valueOf(brs);
				}
			}
		}

		Elements dates = page.getElementsByTag("time");
		for(Element dateEle : dates){
			String val = dateEle.text();
			
			date  = CL_DATE_FORMAT.parse(val);
		}
				
		//return new CLPost(date, link, price, bedrooms, null );
		return null;
	}
	
	public static Document getDocForUrl(String url){
		
		try {
			Thread.sleep(1000);
			return Jsoup.connect(url).get();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			throw new RuntimeException(e);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
	}
}
